{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 - Apache Spark ML - Create train and test set\n",
    "\n",
    "In this chapter, you will:\n",
    "\n",
    "• Create a test and train set\n",
    "\n",
    "• Learn more Spark functionality and how to use it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession \n",
    "\n",
    "spark = SparkSession.builder \\\n",
    "    .master('local[*]') \\\n",
    "    .appName(\"ApacheSparkML\") \\\n",
    "    .getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After filtering and working on the train DataFrame, we need to make sure the test set has the same structure.\n",
    "\n",
    "Load testing data from CSV file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = spark.read.csv(\"testing_bot_data.csv\", header= True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StructType(List(StructField(id,StringType,true),StructField(id_str,StringType,true),StructField(screen_name,StringType,true),StructField(location,StringType,true),StructField(description,StringType,true),StructField(url,StringType,true),StructField(followers_count,StringType,true),StructField(friends_count,StringType,true),StructField(listed_count,StringType,true),StructField(created_at,StringType,true),StructField(favourites_count,StringType,true),StructField(verified,StringType,true),StructField(statuses_count,StringType,true),StructField(lang,StringType,true),StructField(status,StringType,true),StructField(default_profile,StringType,true),StructField(default_profile_image,StringType,true),StructField(has_extended_profile,StringType,true),StructField(name,StringType,true)))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "578"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>id_str</th>\n",
       "      <th>screen_name</th>\n",
       "      <th>location</th>\n",
       "      <th>description</th>\n",
       "      <th>url</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>friends_count</th>\n",
       "      <th>listed_count</th>\n",
       "      <th>created_at</th>\n",
       "      <th>favourites_count</th>\n",
       "      <th>verified</th>\n",
       "      <th>statuses_count</th>\n",
       "      <th>lang</th>\n",
       "      <th>status</th>\n",
       "      <th>default_profile</th>\n",
       "      <th>default_profile_image</th>\n",
       "      <th>has_extended_profile</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2281292622</td>\n",
       "      <td>2281292622</td>\n",
       "      <td>__keating</td>\n",
       "      <td>brooklyn</td>\n",
       "      <td>lgbt editor at @buzzfeed. shannon.keating@buzz...</td>\n",
       "      <td>https://t.co/QneJmYRyhj</td>\n",
       "      <td>4466</td>\n",
       "      <td>1295</td>\n",
       "      <td>111</td>\n",
       "      <td>Tue Jan 07 23:26:52 +0000 2014</td>\n",
       "      <td>1579</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>3036</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'created_at': 'Tue Apr 11 15:31:51 +0000 201...</td>\n",
       "      <td>'truncated': False</td>\n",
       "      <td>'entities': {'hashtags': []</td>\n",
       "      <td>'symbols': []</td>\n",
       "      <td>'user_mentions': [{'screen_name': 'Carrasquillo'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2344040251</td>\n",
       "      <td>2344040251</td>\n",
       "      <td>_callme_Dani</td>\n",
       "      <td>Los Angeles, CA</td>\n",
       "      <td>News Curation Editor @BuzzFeedNews I do a lot ...</td>\n",
       "      <td>None</td>\n",
       "      <td>295</td>\n",
       "      <td>1016</td>\n",
       "      <td>10</td>\n",
       "      <td>Fri Feb 14 19:45:56 +0000 2014</td>\n",
       "      <td>300</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>618</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'created_at': 'Tue Apr 11 00:56:02 +0000 201...</td>\n",
       "      <td>'truncated': False</td>\n",
       "      <td>'entities': {'hashtags': []</td>\n",
       "      <td>'symbols': []</td>\n",
       "      <td>'user_mentions': [{'screen_name': 'elliesunak...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>765871267</td>\n",
       "      <td>765871267</td>\n",
       "      <td>_little_britt_</td>\n",
       "      <td>None</td>\n",
       "      <td>Family comes first! Also I am in love with piz...</td>\n",
       "      <td>https://t.co/E7DE1cJB7e</td>\n",
       "      <td>1001678</td>\n",
       "      <td>3017</td>\n",
       "      <td>14</td>\n",
       "      <td>8/18/2012 15:13</td>\n",
       "      <td>13040</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>3329</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'place': None, 'retweeted': False, 'favorite...</td>\n",
       "      <td>'created_at': 'Sat Apr 08 19:18:41 +0000 2017'</td>\n",
       "      <td>'id': 850790016838238210</td>\n",
       "      <td>'lang': 'en'</td>\n",
       "      <td>'retweet_count': 2037}\"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4772373433</td>\n",
       "      <td>4772373433</td>\n",
       "      <td>134k5</td>\n",
       "      <td>None</td>\n",
       "      <td>@BuzzFeedJapan ��� @cnet_japan / DM��܋�㋁_��_��...</td>\n",
       "      <td>https://t.co/Cbguzs2PjT</td>\n",
       "      <td>445</td>\n",
       "      <td>487</td>\n",
       "      <td>17</td>\n",
       "      <td>Sun Jan 17 07:11:45 +0000 2016</td>\n",
       "      <td>1112</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>46</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{'created_at': 'Sat Apr 08 08:41:08 +0000 201...</td>\n",
       "      <td>'in_reply_to_status_id': 850628293522894849</td>\n",
       "      <td>'in_reply_to_status_id_str': '850628293522894...</td>\n",
       "      <td>'in_reply_to_user_id': 2249898907</td>\n",
       "      <td>'in_reply_to_user_id_str': '2249898907'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1324548560</td>\n",
       "      <td>None</td>\n",
       "      <td>2181chrom_bot</td>\n",
       "      <td>自分の天幕</td>\n",
       "      <td>これはFE覚醒のクロム…つまり俺がツイ廃なbotらしい。よく分からんがネタ要素しかないそうだ...</td>\n",
       "      <td>http://t.co/10Swf6luED</td>\n",
       "      <td>187</td>\n",
       "      <td>68</td>\n",
       "      <td>13</td>\n",
       "      <td>Wed Apr 03 13:00:42 +0000 2013</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>690359</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{u'lang': u'ja', u'text': u'@2181lucina_bot \\...</td>\n",
       "      <td>u'in_reply_to_status_id': 851191070100606976</td>\n",
       "      <td>u'in_reply_to_screen_name': u'2181lucina_bot'</td>\n",
       "      <td>u'id_str': u'851191507486986241'</td>\n",
       "      <td>u'urls': []</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2561341789</td>\n",
       "      <td>None</td>\n",
       "      <td>2LA1R_bot</td>\n",
       "      <td>None</td>\n",
       "      <td>ふれあ語をつぶやくbotです たまに中の人(ふれあ)もつぶやきます</td>\n",
       "      <td>None</td>\n",
       "      <td>80</td>\n",
       "      <td>87</td>\n",
       "      <td>None</td>\n",
       "      <td>Wed Jun 11 13:12:06 +0000 2014</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>20167</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{u'lang': u'ja', u'text': u'\\u3010\\u3075\\u308...</td>\n",
       "      <td>u'id_str': u'851191514206097409'</td>\n",
       "      <td>u'urls': []</td>\n",
       "      <td>u'id': 851191514206097409}\"</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>347810134</td>\n",
       "      <td>None</td>\n",
       "      <td>3pei_bot</td>\n",
       "      <td>三河屋</td>\n",
       "      <td>■ちわーす！三河屋でーす！三郎くんには負けませんｗｗｗ■サザエさんに過去登場、三河屋さんへ勤...</td>\n",
       "      <td>http://twpf.jp/3pei_bot</td>\n",
       "      <td>2020</td>\n",
       "      <td>1978</td>\n",
       "      <td>56</td>\n",
       "      <td>Wed Aug 03 11:52:59 +0000 2011</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>968182</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{u'lang': u'ja', u'text': u'@kazenoraby \\uff6...</td>\n",
       "      <td>u'in_reply_to_status_id': 851191255841124352</td>\n",
       "      <td>u'in_reply_to_screen_name': u'kazenoraby'</td>\n",
       "      <td>u'id_str': u'851191511765176320'</td>\n",
       "      <td>u'urls': []</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>856303860</td>\n",
       "      <td>None</td>\n",
       "      <td>94kichi_bot</td>\n",
       "      <td>None</td>\n",
       "      <td>94 チャック・ウィルソンと愉快な仲間たちの笑いあり涙ありなちょこっとキチガイツイートを集め...</td>\n",
       "      <td>None</td>\n",
       "      <td>70</td>\n",
       "      <td>80</td>\n",
       "      <td>2</td>\n",
       "      <td>Mon Oct 01 12:39:46 +0000 2012</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>76735</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{u'lang': u'ja', u'text': u'\\u307f\\u3093\\u306...</td>\n",
       "      <td>u'id_str': u'851191524062670848'</td>\n",
       "      <td>u'urls': []</td>\n",
       "      <td>u'id': 851191524062670848}\"</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8.32875E+17</td>\n",
       "      <td>None</td>\n",
       "      <td>A3_Dekasegi_bot</td>\n",
       "      <td>ビロード駅前</td>\n",
       "      <td>シトロン「A3!出稼ぎ日誌ダヨー！みんなの出稼ぎ中のあんなことやこんなことをまとめた日誌ネ！...</td>\n",
       "      <td>https://t.co/t171JmIrjL</td>\n",
       "      <td>181</td>\n",
       "      <td>144</td>\n",
       "      <td>2</td>\n",
       "      <td>Sat Feb 18 08:50:03 +0000 2017</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1960</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{u'lang': u'ja', u'text': u'\\u30b7\\u30c8\\u30e...</td>\n",
       "      <td>u'id_str': u'851191498854998016'</td>\n",
       "      <td>u'urls': []</td>\n",
       "      <td>u'id': 851191498854998016}\"</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>88856792</td>\n",
       "      <td>None</td>\n",
       "      <td>aamir_khan</td>\n",
       "      <td>Mumbai</td>\n",
       "      <td>Actor.</td>\n",
       "      <td>https://t.co/l1dUhQjS8Y</td>\n",
       "      <td>20419393</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>Tue Nov 10 05:08:56 +0000 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>468</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{u'lang': u'en', u'text': u'Hey guys, doing s...</td>\n",
       "      <td>u'id_str': u'849903030598344704'</td>\n",
       "      <td>u'urls': []</td>\n",
       "      <td>u'media': [{u'expanded_url': u'https://twitte...</td>\n",
       "      <td>u'display_url': u'pic.twitter.com/uYmd8FKOVH'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>7.13557E+17</td>\n",
       "      <td>7.14E+17</td>\n",
       "      <td>AceWallpaperBot</td>\n",
       "      <td>None</td>\n",
       "      <td>Posts a [SFW] wallpaper every 30 minutes. Ever...</td>\n",
       "      <td>https://t.co/tuq9iPhc0Z</td>\n",
       "      <td>20423</td>\n",
       "      <td>8</td>\n",
       "      <td>130</td>\n",
       "      <td>3/26/2016 2:42</td>\n",
       "      <td>0</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>17387</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'favorite_count': 12, 'id': 8514323558563061...</td>\n",
       "      <td>'contributors': None</td>\n",
       "      <td>'id_str': '851432355856306177'</td>\n",
       "      <td>'in_reply_to_user_id': None</td>\n",
       "      <td>'favorited': False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1566746503</td>\n",
       "      <td>1566746503</td>\n",
       "      <td>acormier_bzfd</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Investigative Reporter-Editor @buzzfeed. Forme...</td>\n",
       "      <td>https://t.co/iiAufe34NG</td>\n",
       "      <td>1436</td>\n",
       "      <td>2017</td>\n",
       "      <td>55</td>\n",
       "      <td>Wed Jul 03 21:28:01 +0000 2013</td>\n",
       "      <td>1099</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>967</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'created_at': 'Tue Apr 11 14:07:03 +0000 201...</td>\n",
       "      <td>'in_reply_to_status_id': None</td>\n",
       "      <td>'in_reply_to_status_id_str': None</td>\n",
       "      <td>'in_reply_to_user_id': None</td>\n",
       "      <td>'in_reply_to_user_id_str': None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>90420314</td>\n",
       "      <td>None</td>\n",
       "      <td>ActuallyNPH</td>\n",
       "      <td>Manhattanland</td>\n",
       "      <td>I dig variety acts, Pixar, puppets, prestidigi...</td>\n",
       "      <td>https://t.co/c8ibVF0Lfq</td>\n",
       "      <td>26376073</td>\n",
       "      <td>91</td>\n",
       "      <td>60767</td>\n",
       "      <td>Mon Nov 16 15:54:48 +0000 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>2666</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{u'lang': u'en', u'text': u\"\"Podcast recommen...</td>\n",
       "      <td>intimate</td>\n",
       "      <td>crazy</td>\n",
       "      <td>mysterious</td>\n",
       "      <td>informative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>184910040</td>\n",
       "      <td>None</td>\n",
       "      <td>Adele</td>\n",
       "      <td>London</td>\n",
       "      <td>https://t.co/5Vf55N5xFe</td>\n",
       "      <td>https://t.co/5Vf55N5xFe</td>\n",
       "      <td>27950120</td>\n",
       "      <td>157</td>\n",
       "      <td>32993</td>\n",
       "      <td>Mon Aug 30 19:53:19 +0000 2010</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>287</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{u'lang': u'en', u'truncated': True, u'text':...</td>\n",
       "      <td>u'id_str': u'844477332693897216'</td>\n",
       "      <td>u'urls': [{u'url': u'https://t.co/W5570kii0M'</td>\n",
       "      <td>u'expanded_url': u'https://twitter.com/i/web/...</td>\n",
       "      <td>u'retweet_count': 5435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>157690631</td>\n",
       "      <td>157690631</td>\n",
       "      <td>aflores</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>Latino affairs and breaking news reporter for ...</td>\n",
       "      <td>http://t.co/ykLlu4DCdC</td>\n",
       "      <td>5210</td>\n",
       "      <td>2555</td>\n",
       "      <td>232</td>\n",
       "      <td>Sun Jun 20 15:27:55 +0000 2010</td>\n",
       "      <td>1169</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>5501</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'created_at': 'Tue Apr 11 18:31:46 +0000 201...</td>\n",
       "      <td>'in_reply_to_status_id': None</td>\n",
       "      <td>'in_reply_to_status_id_str': None</td>\n",
       "      <td>'in_reply_to_user_id': None</td>\n",
       "      <td>'in_reply_to_user_id_str': None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>42420346</td>\n",
       "      <td>None</td>\n",
       "      <td>agnezmo</td>\n",
       "      <td>Los Angeles</td>\n",
       "      <td>DREAM BELIEVE &amp; MAKE IT HAPPEN! Instagram: @ag...</td>\n",
       "      <td>https://t.co/PYhVmN5FYH</td>\n",
       "      <td>16328463</td>\n",
       "      <td>1071</td>\n",
       "      <td>10848</td>\n",
       "      <td>Mon May 25 15:05:00 +0000 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>14596</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{u'lang': u'en', u'text': u'Too comfortable \\...</td>\n",
       "      <td>u'id_str': u'842052863853531137'</td>\n",
       "      <td>u'urls': [{u'url': u'https://t.co/heDL0FPSqO'</td>\n",
       "      <td>u'expanded_url': u'https://www.instagram.com/...</td>\n",
       "      <td>u'retweet_count': 211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>42382447</td>\n",
       "      <td>42382447</td>\n",
       "      <td>Agostinozoida</td>\n",
       "      <td>Agostinozoida</td>\n",
       "      <td>Agostinozoida</td>\n",
       "      <td>https://t.co/2gdWyQjNq4</td>\n",
       "      <td>964</td>\n",
       "      <td>331</td>\n",
       "      <td>15</td>\n",
       "      <td>5/25/2009 10:02</td>\n",
       "      <td>2832</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>6389</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'place': None, 'retweeted_status': {'place':...</td>\n",
       "      <td>Jamie Kennedy</td>\n",
       "      <td>Kate Quigley</td>\n",
       "      <td>K-Von and more! Hope you can make it! https:/...</td>\n",
       "      <td>'extended_entities': {'media': [{'id_str': '8...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>43993280</td>\n",
       "      <td>43993280</td>\n",
       "      <td>AhmedBharoocha</td>\n",
       "      <td>None</td>\n",
       "      <td>I forget to tweet</td>\n",
       "      <td>https://t.co/iM3lj0uBHg</td>\n",
       "      <td>5254</td>\n",
       "      <td>2407</td>\n",
       "      <td>102</td>\n",
       "      <td>6/1/2009 22:44</td>\n",
       "      <td>3082</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>3392</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'place': None, 'retweeted_status': {'place':...</td>\n",
       "      <td>'possibly_sensitive': False</td>\n",
       "      <td>'created_at': 'Wed Apr 05 14:33:06 +0000 2017'</td>\n",
       "      <td>'id': 849630982311366656</td>\n",
       "      <td>'lang': 'en'</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2305236733</td>\n",
       "      <td>None</td>\n",
       "      <td>aho_levi_bot</td>\n",
       "      <td>teamリヴァイ</td>\n",
       "      <td>YDK。botシステム外部レンタルから自作に乗り換えました。呼ぶとヒョッコリ現れます。モノを...</td>\n",
       "      <td>None</td>\n",
       "      <td>446</td>\n",
       "      <td>431</td>\n",
       "      <td>4</td>\n",
       "      <td>Wed Jan 22 17:52:19 +0000 2014</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>119680</td>\n",
       "      <td>ja</td>\n",
       "      <td>\"{u'lang': u'ja', u'text': u'@kotsubu_149 \\u30...</td>\n",
       "      <td>u'in_reply_to_status_id': 851191318789226497</td>\n",
       "      <td>u'in_reply_to_screen_name': u'kotsubu_149'</td>\n",
       "      <td>u'id_str': u'851191500176211970'</td>\n",
       "      <td>u'urls': []</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>7.42794E+17</td>\n",
       "      <td>7.43E+17</td>\n",
       "      <td>aiomacbot</td>\n",
       "      <td>None</td>\n",
       "      <td>A simple all-in-one bot for the Mac</td>\n",
       "      <td>https://t.co/cR0QCvj4cu</td>\n",
       "      <td>1404</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6/14/2016 18:59</td>\n",
       "      <td>12</td>\n",
       "      <td>FALSE</td>\n",
       "      <td>141</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'favorite_count': 2, 'id': 85117552838408192...</td>\n",
       "      <td>'contributors': None</td>\n",
       "      <td>'id_str': '851175528384081922'</td>\n",
       "      <td>'in_reply_to_user_id': 1090862568</td>\n",
       "      <td>'favorited': False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>31348594</td>\n",
       "      <td>None</td>\n",
       "      <td>akshaykumar</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>https://t.co/nVpOquecxs</td>\n",
       "      <td>16933838</td>\n",
       "      <td>21</td>\n",
       "      <td>14682</td>\n",
       "      <td>Wed Apr 15 05:23:19 +0000 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>3673</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{u'lang': u'en', u'truncated': True, u'text':...</td>\n",
       "      <td>u'id_str': u'851080708130983937'</td>\n",
       "      <td>u'urls': [{u'url': u'https://t.co/fNgLENylXk'</td>\n",
       "      <td>u'expanded_url': u'https://twitter.com/i/web/...</td>\n",
       "      <td>u'retweet_count': 1725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>122085859</td>\n",
       "      <td>122085859</td>\n",
       "      <td>AlannaBennett</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>Writer, all sorts. Film &amp; TV Writer @BuzzFeedN...</td>\n",
       "      <td>https://t.co/nLwRgOwrQ9</td>\n",
       "      <td>8619</td>\n",
       "      <td>3229</td>\n",
       "      <td>190</td>\n",
       "      <td>Thu Mar 11 15:00:48 +0000 2010</td>\n",
       "      <td>50921</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>38918</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'created_at': 'Tue Apr 11 19:59:24 +0000 201...</td>\n",
       "      <td>'truncated': False</td>\n",
       "      <td>'entities': {'hashtags': []</td>\n",
       "      <td>'symbols': []</td>\n",
       "      <td>'user_mentions': []</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>23573083</td>\n",
       "      <td>None</td>\n",
       "      <td>AlArabiya_Brk</td>\n",
       "      <td>Dubai</td>\n",
       "      <td>خدمة قناة #العربية للأخبار العاجلة</td>\n",
       "      <td>https://t.co/3J8D8Z8yvo</td>\n",
       "      <td>15610695</td>\n",
       "      <td>36</td>\n",
       "      <td>15248</td>\n",
       "      <td>Tue Mar 10 09:15:09 +0000 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>82958</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{u'lang': u'ar', u'text': u'\\u0635\\u062d\\u064...</td>\n",
       "      <td>u'id_str': u'851171829091848196'</td>\n",
       "      <td>u'urls': []</td>\n",
       "      <td>u'retweet_count': 26</td>\n",
       "      <td>u'id': 851171829091848196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>43152482</td>\n",
       "      <td>None</td>\n",
       "      <td>AlejandroSanz</td>\n",
       "      <td>None</td>\n",
       "      <td>https://t.co/sS9G4mmPM0</td>\n",
       "      <td>https://t.co/jh99bUQM9w</td>\n",
       "      <td>15836207</td>\n",
       "      <td>1046</td>\n",
       "      <td>32067</td>\n",
       "      <td>Thu May 28 17:21:35 +0000 2009</td>\n",
       "      <td>None</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>27660</td>\n",
       "      <td>es</td>\n",
       "      <td>\"{u'lang': u'es', u'retweeted_status': {u'lang...</td>\n",
       "      <td>u'id_str': u'851163510331629568'</td>\n",
       "      <td>u'urls': [{u'url': u'https://t.co/MGqSQhRAwB'</td>\n",
       "      <td>u'expanded_url': u'https://twitter.com/alejan...</td>\n",
       "      <td>u'retweet_count': 214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>188857501</td>\n",
       "      <td>188857501</td>\n",
       "      <td>alexis_levinson</td>\n",
       "      <td>Washington, D.C.</td>\n",
       "      <td>@BuzzFeedNews reporter. former @NRO. Scared of...</td>\n",
       "      <td>None</td>\n",
       "      <td>14749</td>\n",
       "      <td>1843</td>\n",
       "      <td>584</td>\n",
       "      <td>Thu Sep 09 19:19:59 +0000 2010</td>\n",
       "      <td>2389</td>\n",
       "      <td>TRUE</td>\n",
       "      <td>20076</td>\n",
       "      <td>en</td>\n",
       "      <td>\"{'created_at': 'Tue Apr 11 17:20:13 +0000 201...</td>\n",
       "      <td>'truncated': False</td>\n",
       "      <td>'entities': {'hashtags': []</td>\n",
       "      <td>'symbols': []</td>\n",
       "      <td>'user_mentions': [{'screen_name': 'jmartNYT'</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id      id_str      screen_name          location  \\\n",
       "0    2281292622  2281292622        __keating          brooklyn   \n",
       "1    2344040251  2344040251     _callme_Dani   Los Angeles, CA   \n",
       "2     765871267   765871267   _little_britt_              None   \n",
       "3    4772373433  4772373433            134k5              None   \n",
       "4    1324548560        None    2181chrom_bot             自分の天幕   \n",
       "5    2561341789        None        2LA1R_bot              None   \n",
       "6     347810134        None         3pei_bot               三河屋   \n",
       "7     856303860        None      94kichi_bot              None   \n",
       "8   8.32875E+17        None  A3_Dekasegi_bot            ビロード駅前   \n",
       "9      88856792        None       aamir_khan            Mumbai   \n",
       "10  7.13557E+17    7.14E+17  AceWallpaperBot              None   \n",
       "11   1566746503  1566746503    acormier_bzfd      New York, NY   \n",
       "12     90420314        None      ActuallyNPH     Manhattanland   \n",
       "13    184910040        None            Adele            London   \n",
       "14    157690631   157690631          aflores       Los Angeles   \n",
       "15     42420346        None          agnezmo       Los Angeles   \n",
       "16     42382447    42382447    Agostinozoida     Agostinozoida   \n",
       "17     43993280    43993280   AhmedBharoocha              None   \n",
       "18   2305236733        None     aho_levi_bot          teamリヴァイ   \n",
       "19  7.42794E+17    7.43E+17        aiomacbot              None   \n",
       "20     31348594        None      akshaykumar              None   \n",
       "21    122085859   122085859    AlannaBennett      New York, NY   \n",
       "22     23573083        None    AlArabiya_Brk             Dubai   \n",
       "23     43152482        None    AlejandroSanz              None   \n",
       "24    188857501   188857501  alexis_levinson  Washington, D.C.   \n",
       "\n",
       "                                          description  \\\n",
       "0   lgbt editor at @buzzfeed. shannon.keating@buzz...   \n",
       "1   News Curation Editor @BuzzFeedNews I do a lot ...   \n",
       "2   Family comes first! Also I am in love with piz...   \n",
       "3   @BuzzFeedJapan ��� @cnet_japan / DM��܋�㋁_��_��...   \n",
       "4   これはFE覚醒のクロム…つまり俺がツイ廃なbotらしい。よく分からんがネタ要素しかないそうだ...   \n",
       "5                   ふれあ語をつぶやくbotです たまに中の人(ふれあ)もつぶやきます   \n",
       "6   ■ちわーす！三河屋でーす！三郎くんには負けませんｗｗｗ■サザエさんに過去登場、三河屋さんへ勤...   \n",
       "7   94 チャック・ウィルソンと愉快な仲間たちの笑いあり涙ありなちょこっとキチガイツイートを集め...   \n",
       "8   シトロン「A3!出稼ぎ日誌ダヨー！みんなの出稼ぎ中のあんなことやこんなことをまとめた日誌ネ！...   \n",
       "9                                              Actor.   \n",
       "10  Posts a [SFW] wallpaper every 30 minutes. Ever...   \n",
       "11  Investigative Reporter-Editor @buzzfeed. Forme...   \n",
       "12  I dig variety acts, Pixar, puppets, prestidigi...   \n",
       "13                            https://t.co/5Vf55N5xFe   \n",
       "14  Latino affairs and breaking news reporter for ...   \n",
       "15  DREAM BELIEVE & MAKE IT HAPPEN! Instagram: @ag...   \n",
       "16                                      Agostinozoida   \n",
       "17                                  I forget to tweet   \n",
       "18  YDK。botシステム外部レンタルから自作に乗り換えました。呼ぶとヒョッコリ現れます。モノを...   \n",
       "19                A simple all-in-one bot for the Mac   \n",
       "20                                               None   \n",
       "21  Writer, all sorts. Film & TV Writer @BuzzFeedN...   \n",
       "22                 خدمة قناة #العربية للأخبار العاجلة   \n",
       "23                            https://t.co/sS9G4mmPM0   \n",
       "24  @BuzzFeedNews reporter. former @NRO. Scared of...   \n",
       "\n",
       "                        url followers_count friends_count listed_count  \\\n",
       "0   https://t.co/QneJmYRyhj            4466          1295          111   \n",
       "1                      None             295          1016           10   \n",
       "2   https://t.co/E7DE1cJB7e         1001678          3017           14   \n",
       "3   https://t.co/Cbguzs2PjT             445           487           17   \n",
       "4    http://t.co/10Swf6luED             187            68           13   \n",
       "5                      None              80            87         None   \n",
       "6   http://twpf.jp/3pei_bot            2020          1978           56   \n",
       "7                      None              70            80            2   \n",
       "8   https://t.co/t171JmIrjL             181           144            2   \n",
       "9   https://t.co/l1dUhQjS8Y        20419393             9            6   \n",
       "10  https://t.co/tuq9iPhc0Z           20423             8          130   \n",
       "11  https://t.co/iiAufe34NG            1436          2017           55   \n",
       "12  https://t.co/c8ibVF0Lfq        26376073            91        60767   \n",
       "13  https://t.co/5Vf55N5xFe        27950120           157        32993   \n",
       "14   http://t.co/ykLlu4DCdC            5210          2555          232   \n",
       "15  https://t.co/PYhVmN5FYH        16328463          1071        10848   \n",
       "16  https://t.co/2gdWyQjNq4             964           331           15   \n",
       "17  https://t.co/iM3lj0uBHg            5254          2407          102   \n",
       "18                     None             446           431            4   \n",
       "19  https://t.co/cR0QCvj4cu            1404             0            2   \n",
       "20  https://t.co/nVpOquecxs        16933838            21        14682   \n",
       "21  https://t.co/nLwRgOwrQ9            8619          3229          190   \n",
       "22  https://t.co/3J8D8Z8yvo        15610695            36        15248   \n",
       "23  https://t.co/jh99bUQM9w        15836207          1046        32067   \n",
       "24                     None           14749          1843          584   \n",
       "\n",
       "                        created_at favourites_count verified statuses_count  \\\n",
       "0   Tue Jan 07 23:26:52 +0000 2014             1579     TRUE           3036   \n",
       "1   Fri Feb 14 19:45:56 +0000 2014              300    FALSE            618   \n",
       "2                  8/18/2012 15:13            13040     TRUE           3329   \n",
       "3   Sun Jan 17 07:11:45 +0000 2016             1112    FALSE             46   \n",
       "4   Wed Apr 03 13:00:42 +0000 2013             None     None         690359   \n",
       "5   Wed Jun 11 13:12:06 +0000 2014             None     None          20167   \n",
       "6   Wed Aug 03 11:52:59 +0000 2011             None     None         968182   \n",
       "7   Mon Oct 01 12:39:46 +0000 2012             None     None          76735   \n",
       "8   Sat Feb 18 08:50:03 +0000 2017             None     None           1960   \n",
       "9   Tue Nov 10 05:08:56 +0000 2009             None     TRUE            468   \n",
       "10                  3/26/2016 2:42                0    FALSE          17387   \n",
       "11  Wed Jul 03 21:28:01 +0000 2013             1099     TRUE            967   \n",
       "12  Mon Nov 16 15:54:48 +0000 2009             None     TRUE           2666   \n",
       "13  Mon Aug 30 19:53:19 +0000 2010             None     TRUE            287   \n",
       "14  Sun Jun 20 15:27:55 +0000 2010             1169     TRUE           5501   \n",
       "15  Mon May 25 15:05:00 +0000 2009             None     TRUE          14596   \n",
       "16                 5/25/2009 10:02             2832    FALSE           6389   \n",
       "17                  6/1/2009 22:44             3082    FALSE           3392   \n",
       "18  Wed Jan 22 17:52:19 +0000 2014             None     None         119680   \n",
       "19                 6/14/2016 18:59               12    FALSE            141   \n",
       "20  Wed Apr 15 05:23:19 +0000 2009             None     TRUE           3673   \n",
       "21  Thu Mar 11 15:00:48 +0000 2010            50921     TRUE          38918   \n",
       "22  Tue Mar 10 09:15:09 +0000 2009             None     TRUE          82958   \n",
       "23  Thu May 28 17:21:35 +0000 2009             None     TRUE          27660   \n",
       "24  Thu Sep 09 19:19:59 +0000 2010             2389     TRUE          20076   \n",
       "\n",
       "   lang                                             status  \\\n",
       "0    en  \"{'created_at': 'Tue Apr 11 15:31:51 +0000 201...   \n",
       "1    en  \"{'created_at': 'Tue Apr 11 00:56:02 +0000 201...   \n",
       "2    en  \"{'place': None, 'retweeted': False, 'favorite...   \n",
       "3    ja  \"{'created_at': 'Sat Apr 08 08:41:08 +0000 201...   \n",
       "4    ja  \"{u'lang': u'ja', u'text': u'@2181lucina_bot \\...   \n",
       "5    ja  \"{u'lang': u'ja', u'text': u'\\u3010\\u3075\\u308...   \n",
       "6    ja  \"{u'lang': u'ja', u'text': u'@kazenoraby \\uff6...   \n",
       "7    ja  \"{u'lang': u'ja', u'text': u'\\u307f\\u3093\\u306...   \n",
       "8    ja  \"{u'lang': u'ja', u'text': u'\\u30b7\\u30c8\\u30e...   \n",
       "9    en  \"{u'lang': u'en', u'text': u'Hey guys, doing s...   \n",
       "10   en  \"{'favorite_count': 12, 'id': 8514323558563061...   \n",
       "11   en  \"{'created_at': 'Tue Apr 11 14:07:03 +0000 201...   \n",
       "12   en  \"{u'lang': u'en', u'text': u\"\"Podcast recommen...   \n",
       "13   en  \"{u'lang': u'en', u'truncated': True, u'text':...   \n",
       "14   en  \"{'created_at': 'Tue Apr 11 18:31:46 +0000 201...   \n",
       "15   en  \"{u'lang': u'en', u'text': u'Too comfortable \\...   \n",
       "16   en  \"{'place': None, 'retweeted_status': {'place':...   \n",
       "17   en  \"{'place': None, 'retweeted_status': {'place':...   \n",
       "18   ja  \"{u'lang': u'ja', u'text': u'@kotsubu_149 \\u30...   \n",
       "19   en  \"{'favorite_count': 2, 'id': 85117552838408192...   \n",
       "20   en  \"{u'lang': u'en', u'truncated': True, u'text':...   \n",
       "21   en  \"{'created_at': 'Tue Apr 11 19:59:24 +0000 201...   \n",
       "22   en  \"{u'lang': u'ar', u'text': u'\\u0635\\u062d\\u064...   \n",
       "23   es  \"{u'lang': u'es', u'retweeted_status': {u'lang...   \n",
       "24   en  \"{'created_at': 'Tue Apr 11 17:20:13 +0000 201...   \n",
       "\n",
       "                                    default_profile  \\\n",
       "0                                'truncated': False   \n",
       "1                                'truncated': False   \n",
       "2    'created_at': 'Sat Apr 08 19:18:41 +0000 2017'   \n",
       "3       'in_reply_to_status_id': 850628293522894849   \n",
       "4      u'in_reply_to_status_id': 851191070100606976   \n",
       "5                  u'id_str': u'851191514206097409'   \n",
       "6      u'in_reply_to_status_id': 851191255841124352   \n",
       "7                  u'id_str': u'851191524062670848'   \n",
       "8                  u'id_str': u'851191498854998016'   \n",
       "9                  u'id_str': u'849903030598344704'   \n",
       "10                             'contributors': None   \n",
       "11                    'in_reply_to_status_id': None   \n",
       "12                                         intimate   \n",
       "13                 u'id_str': u'844477332693897216'   \n",
       "14                    'in_reply_to_status_id': None   \n",
       "15                 u'id_str': u'842052863853531137'   \n",
       "16                                    Jamie Kennedy   \n",
       "17                      'possibly_sensitive': False   \n",
       "18     u'in_reply_to_status_id': 851191318789226497   \n",
       "19                             'contributors': None   \n",
       "20                 u'id_str': u'851080708130983937'   \n",
       "21                               'truncated': False   \n",
       "22                 u'id_str': u'851171829091848196'   \n",
       "23                 u'id_str': u'851163510331629568'   \n",
       "24                               'truncated': False   \n",
       "\n",
       "                                default_profile_image  \\\n",
       "0                         'entities': {'hashtags': []   \n",
       "1                         'entities': {'hashtags': []   \n",
       "2                            'id': 850790016838238210   \n",
       "3    'in_reply_to_status_id_str': '850628293522894...   \n",
       "4       u'in_reply_to_screen_name': u'2181lucina_bot'   \n",
       "5                                         u'urls': []   \n",
       "6           u'in_reply_to_screen_name': u'kazenoraby'   \n",
       "7                                         u'urls': []   \n",
       "8                                         u'urls': []   \n",
       "9                                         u'urls': []   \n",
       "10                     'id_str': '851432355856306177'   \n",
       "11                  'in_reply_to_status_id_str': None   \n",
       "12                                              crazy   \n",
       "13      u'urls': [{u'url': u'https://t.co/W5570kii0M'   \n",
       "14                  'in_reply_to_status_id_str': None   \n",
       "15      u'urls': [{u'url': u'https://t.co/heDL0FPSqO'   \n",
       "16                                       Kate Quigley   \n",
       "17     'created_at': 'Wed Apr 05 14:33:06 +0000 2017'   \n",
       "18         u'in_reply_to_screen_name': u'kotsubu_149'   \n",
       "19                     'id_str': '851175528384081922'   \n",
       "20      u'urls': [{u'url': u'https://t.co/fNgLENylXk'   \n",
       "21                        'entities': {'hashtags': []   \n",
       "22                                        u'urls': []   \n",
       "23      u'urls': [{u'url': u'https://t.co/MGqSQhRAwB'   \n",
       "24                        'entities': {'hashtags': []   \n",
       "\n",
       "                                 has_extended_profile  \\\n",
       "0                                       'symbols': []   \n",
       "1                                       'symbols': []   \n",
       "2                                        'lang': 'en'   \n",
       "3                   'in_reply_to_user_id': 2249898907   \n",
       "4                    u'id_str': u'851191507486986241'   \n",
       "5                         u'id': 851191514206097409}\"   \n",
       "6                    u'id_str': u'851191511765176320'   \n",
       "7                         u'id': 851191524062670848}\"   \n",
       "8                         u'id': 851191498854998016}\"   \n",
       "9    u'media': [{u'expanded_url': u'https://twitte...   \n",
       "10                        'in_reply_to_user_id': None   \n",
       "11                        'in_reply_to_user_id': None   \n",
       "12                                         mysterious   \n",
       "13   u'expanded_url': u'https://twitter.com/i/web/...   \n",
       "14                        'in_reply_to_user_id': None   \n",
       "15   u'expanded_url': u'https://www.instagram.com/...   \n",
       "16   K-Von and more! Hope you can make it! https:/...   \n",
       "17                           'id': 849630982311366656   \n",
       "18                   u'id_str': u'851191500176211970'   \n",
       "19                  'in_reply_to_user_id': 1090862568   \n",
       "20   u'expanded_url': u'https://twitter.com/i/web/...   \n",
       "21                                      'symbols': []   \n",
       "22                               u'retweet_count': 26   \n",
       "23   u'expanded_url': u'https://twitter.com/alejan...   \n",
       "24                                      'symbols': []   \n",
       "\n",
       "                                                 name  \n",
       "0    'user_mentions': [{'screen_name': 'Carrasquillo'  \n",
       "1    'user_mentions': [{'screen_name': 'elliesunak...  \n",
       "2                             'retweet_count': 2037}\"  \n",
       "3             'in_reply_to_user_id_str': '2249898907'  \n",
       "4                                         u'urls': []  \n",
       "5                                                TRUE  \n",
       "6                                         u'urls': []  \n",
       "7                                                TRUE  \n",
       "8                                                TRUE  \n",
       "9       u'display_url': u'pic.twitter.com/uYmd8FKOVH'  \n",
       "10                                 'favorited': False  \n",
       "11                    'in_reply_to_user_id_str': None  \n",
       "12                                        informative  \n",
       "13                             u'retweet_count': 5435  \n",
       "14                    'in_reply_to_user_id_str': None  \n",
       "15                              u'retweet_count': 211  \n",
       "16   'extended_entities': {'media': [{'id_str': '8...  \n",
       "17                                       'lang': 'en'  \n",
       "18                                        u'urls': []  \n",
       "19                                 'favorited': False  \n",
       "20                             u'retweet_count': 1725  \n",
       "21                                'user_mentions': []  \n",
       "22                          u'id': 851171829091848196  \n",
       "23                              u'retweet_count': 214  \n",
       "24       'user_mentions': [{'screen_name': 'jmartNYT'  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.limit(25) .toPandas ()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clean and prep testing data as well:\n",
    "Remember that here we don't have bot value.\n",
    "\n",
    "You will not drop id, as we will use it to compare results later.\n",
    "\n",
    "Excecute the next commands:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import IntegerType, ArrayType, BooleanType, StringType\n",
    "from pyspark.sql.functions import udf\n",
    "from pyspark.sql.functions import when\n",
    "\n",
    "\n",
    "# Dropping irrelevant columns and duplicates\n",
    "df_test = df_test.drop('default_profile_image','has_extended_profile','url','created_at','lang','id','id_str')\n",
    "df_test = df_test.dropDuplicates()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# First Transformation\n",
    "df_test = df_test.withColumn(\"friends_count\", df_test[\"friends_count\"].cast(IntegerType()))\n",
    "df_test = df_test.withColumn(\"listed_count\", df_test[\"listed_count\"].cast(IntegerType()))\n",
    "df_test = df_test.withColumn(\"favourites_count\", df_test[\"favourites_count\"].cast(IntegerType()))\n",
    "df_test = df_test.withColumn(\"statuses_count\", df_test[\"statuses_count\"].cast(IntegerType()))\n",
    "df_test = df_test.withColumn(\"verified\", df_test[\"verified\"].cast(BooleanType()))\n",
    "df_test = df_test.withColumn(\"default_profile\", df_test[\"default_profile\"].cast(BooleanType()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Second Transformation\n",
    "df_test = df_test.withColumn('default_profile',df_test['default_profile'].cast(IntegerType()))\n",
    "df_test = df_test.withColumn('name',when(df_test['name'].isNull(),0).otherwise(1))\n",
    "df_test = df_test.withColumn('verified',df_test['verified'].cast(IntegerType()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Theird Transformation\n",
    "df_test = df_test.withColumn('verified',when(df_test['verified'].isNull(),0).otherwise(df_test['verified']))\n",
    "df_test = df_test.withColumn('default_profile',when(df_test['default_profile'].isNull(),0).otherwise(df_test['default_profile']))\n",
    "df_test = df_test.withColumn('location',when(df_test['location'].isNull(),0).otherwise(1))\n",
    "df_test = df_test.withColumn('status',when(df_test['status'].isNull(),0).otherwise(1))\n",
    "df_test = df_test.withColumn('screen_name',when(df_test['screen_name'].isNull(),0).otherwise(1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Forth Transformation\n",
    "df_test = df_test.dropna(subset=['description'])\n",
    "\n",
    "def split_and_set(some_str):\n",
    "    if isinstance(some_str, str):\n",
    "        some_str = ''.join(c for c in some_str if c not in \"[](){}<>,'/.\")\n",
    "        return list(set(some_str.split(' ')))\n",
    "    return some_str\n",
    "\n",
    "list_udf = udf(lambda y: split_and_set(y), ArrayType(StringType()))\n",
    "df_test = df_test.withColumn('description', list_udf(df_test['description']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "# Fifth Transformation - fill NA:\n",
    "df_test = df_test.fillna({'followers_count':0,'statuses_count':0,'favourites_count':0,'listed_count':0,'friends_count':0,})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save to parquet:\n",
    "\n",
    "Code sample:\n",
    "```python\n",
    "df_test.write.parquet(\"test_data\")\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.write.parquet(\"test_data\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `test_data` file that you save doesn't consist of information about bots at all.\n",
    "\n",
    "We can use it to compare various algorithms and see how they behave.\n",
    "However, since our training data is supervised, we would like to test it with classified data.\n",
    "This will help us estimate our model.\n",
    "\n",
    "Hence, you will split the training data into testing and train data set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the train data:\n",
    "df_train = spark.read.parquet(\"final_train_data\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split the training data into training and test sets, hold 30% out for testing.\n",
    "\n",
    "Use randomSplit function:\n",
    "\n",
    "```python\n",
    "(trainingData, testData) = some_data.randomSplit((0.7, 0.3))\n",
    "```\n",
    "\n",
    "<details><summary>Hint</summary>\n",
    "<p>\n",
    "\n",
    "Use randomSplit function:\n",
    "    \n",
    "```python\n",
    "(trainingData, testData) = data.randomSplit((0.7, 0.3))\n",
    "\n",
    "```  \n",
    "</p>\n",
    "</details>\n",
    "\n",
    "Remember to validate yourself with count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "(trainingData, testData) = df_train.randomSplit([0.7, 0.3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1714"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainingData.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "719"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testData.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the split data for the next Chapter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "testData.write.mode('overwrite').parquet(\"classified_test_data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainingData.write.mode('overwrite').parquet(\"classified_train_data\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Well Done! 👏👏👏\n",
    "\n",
    "\n",
    "## You just finished:  Apache Spark ML - Create train and test set \n",
    "\n",
    "\n",
    "## Next exercise: Apache Spark ML and create machine learning models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
